{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. 特征工程：20分\n",
    "2. 聚类：30分\n",
    "3. CH_Score计算：30分\n",
    "4. 画图：10分\n",
    "5. 保存结果到文件：10分"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 特征工程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 读取数据\n",
    "train = pd.read_csv('./users.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 38209 entries, 0 to 38208\n",
      "Data columns (total 7 columns):\n",
      "user_id      38209 non-null int64\n",
      "locale       38209 non-null object\n",
      "birthyear    38209 non-null object\n",
      "gender       38100 non-null object\n",
      "joinedAt     38152 non-null object\n",
      "location     32745 non-null object\n",
      "timezone     37773 non-null float64\n",
      "dtypes: float64(1), int64(1), object(5)\n",
      "memory usage: 2.0+ MB\n"
     ]
    }
   ],
   "source": [
    "# 查看数据类型\n",
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>locale</th>\n",
       "      <th>birthyear</th>\n",
       "      <th>gender</th>\n",
       "      <th>joinedAt</th>\n",
       "      <th>location</th>\n",
       "      <th>timezone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3197468391</td>\n",
       "      <td>id_ID</td>\n",
       "      <td>1993</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-10-02T06:40:55.524Z</td>\n",
       "      <td>Medan  Indonesia</td>\n",
       "      <td>480.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3537982273</td>\n",
       "      <td>id_ID</td>\n",
       "      <td>1992</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-09-29T18:03:12.111Z</td>\n",
       "      <td>Medan  Indonesia</td>\n",
       "      <td>420.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>823183725</td>\n",
       "      <td>en_US</td>\n",
       "      <td>1975</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-10-06T03:14:07.149Z</td>\n",
       "      <td>Stratford  Ontario</td>\n",
       "      <td>-240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1872223848</td>\n",
       "      <td>en_US</td>\n",
       "      <td>1991</td>\n",
       "      <td>female</td>\n",
       "      <td>2012-11-04T08:59:43.783Z</td>\n",
       "      <td>Tehran  Iran</td>\n",
       "      <td>210.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3429017717</td>\n",
       "      <td>id_ID</td>\n",
       "      <td>1995</td>\n",
       "      <td>female</td>\n",
       "      <td>2012-09-10T16:06:53.132Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>420.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      user_id locale birthyear  gender                  joinedAt  \\\n",
       "0  3197468391  id_ID      1993    male  2012-10-02T06:40:55.524Z   \n",
       "1  3537982273  id_ID      1992    male  2012-09-29T18:03:12.111Z   \n",
       "2   823183725  en_US      1975    male  2012-10-06T03:14:07.149Z   \n",
       "3  1872223848  en_US      1991  female  2012-11-04T08:59:43.783Z   \n",
       "4  3429017717  id_ID      1995  female  2012-09-10T16:06:53.132Z   \n",
       "\n",
       "             location  timezone  \n",
       "0    Medan  Indonesia     480.0  \n",
       "1    Medan  Indonesia     420.0  \n",
       "2  Stratford  Ontario    -240.0  \n",
       "3        Tehran  Iran     210.0  \n",
       "4                 NaN     420.0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看数据前 5 行\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>locale</th>\n",
       "      <th>birthyear</th>\n",
       "      <th>gender</th>\n",
       "      <th>joinedAt</th>\n",
       "      <th>timezone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id_ID</td>\n",
       "      <td>1993</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-10-02T06:40:55.524Z</td>\n",
       "      <td>480.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id_ID</td>\n",
       "      <td>1992</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-09-29T18:03:12.111Z</td>\n",
       "      <td>420.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>en_US</td>\n",
       "      <td>1975</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-10-06T03:14:07.149Z</td>\n",
       "      <td>-240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>en_US</td>\n",
       "      <td>1991</td>\n",
       "      <td>female</td>\n",
       "      <td>2012-11-04T08:59:43.783Z</td>\n",
       "      <td>210.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id_ID</td>\n",
       "      <td>1995</td>\n",
       "      <td>female</td>\n",
       "      <td>2012-09-10T16:06:53.132Z</td>\n",
       "      <td>420.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  locale birthyear  gender                  joinedAt  timezone\n",
       "0  id_ID      1993    male  2012-10-02T06:40:55.524Z     480.0\n",
       "1  id_ID      1992    male  2012-09-29T18:03:12.111Z     420.0\n",
       "2  en_US      1975    male  2012-10-06T03:14:07.149Z    -240.0\n",
       "3  en_US      1991  female  2012-11-04T08:59:43.783Z     210.0\n",
       "4  id_ID      1995  female  2012-09-10T16:06:53.132Z     420.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 去除没用的特征\n",
    "train = train.drop(['user_id', 'location'], axis=1)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对每个属性值进行观察\n",
    "def col_check(datas, col):\n",
    "    unique = set()\n",
    "    for data in datas[col]:\n",
    "        unique.add(data)\n",
    "    print unique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "locale:\n",
      "set(['bg_BG', 'zh_HK', 'es_LA', 'sk_SK', 'fr_FR', 'fi_FI', 'hi_IN', 'pt_PT', 'it_IT', 'et_EE', 'sr_RS', 'vi_VN', 'uk_UA', 'ar_AR', 'nl_NL', 'km_KH', 'en_US', 'lv_LV', 'el_GR', 'cy_GB', 'th_TH', 'bn_IN', 'ro_RO', 'en_PI', 'ca_ES', 'az_AZ', 'id_ID', 'ka_GE', 'hu_HU', 'ja_JP', 'lt_LT', 'hr_HR', 'es_ES', 'en_GB', 'ru_RU', 'en_UD', 'ko_KR', 'pt_BR', 'da_DK', 'zh_TW', 'pa_IN', 'cs_CZ', 'ms_MY', 'ku_TR', 'sq_AL', 'tr_TR', 'de_DE', 'es_MX', 'mk_MK', 'en_IN', 'af_ZA', 'fr_CA', 'bs_BA', 'fa_IR', 'zh_CN', 'mn_MN', 'tl_PH', 'eo_EO', 'fb_LT', 'sv_SE', 'he_IL', 'pl_PL', 'nb_NO', 'jv_ID'])\n",
      "birthyear:\n",
      "set(['1948', '1949', '1942', '1943', '1940', '1941', '1946', '1947', '1944', '1945', '23-May', '1955', '1954', '1957', '1956', '1951', '1950', '1953', '1952', '1959', '1958', '1920', '1921', '1922', '1923', '1924', '1927', '1928', '1929', 'None', '1933', '1932', '1931', '1930', '1937', '1936', '1935', '1934', '1939', '1938', '1908', '1909', '1906', '1907', '1905', '1902', '1903', '1900', '1986', '1987', '1984', '1985', '1982', '1983', '1980', '1981', '1988', '1989', '1919', '1911', '1910', '1913', '1912', '1915', '1914', '1917', '1916', '1991', '1990', '1993', '1992', '1995', '1994', '1997', '1996', '1999', '1998', '1968', '1969', '1964', '1965', '1966', '1967', '1960', '1961', '1962', '1963', '1979', '1978', '1977', '1976', '1975', '1974', '1973', '1972', '1971', '1970', '16-Mar'])\n",
      "gender:\n",
      "set([nan, 'male', 'female'])\n",
      "timezone:\n",
      "set([0.0, nan, nan, nan, nan, nan, nan, nan, 30.0, nan, 690.0, nan, nan, nan, 60.0, nan, nan, nan, 90.0, nan, nan, nan, nan, nan, 120.0, nan, nan, nan, nan, nan, nan, nan, 150.0, nan, nan, nan, nan, nan, nan, nan, nan, 180.0, nan, nan, nan, nan, nan, nan, nan, 210.0, nan, nan, nan, nan, nan, nan, nan, 240.0, nan, nan, nan, nan, nan, nan, 270.0, nan, nan, nan, nan, nan, 300.0, nan, nan, nan, nan, nan, nan, 330.0, nan, nan, 360.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, 390.0, nan, nan, nan, nan, nan, 420.0, nan, nan, nan, nan, nan, 450.0, nan, nan, nan, 480.0, nan, nan, nan, nan, 510.0, nan, nan, nan, nan, nan, nan, nan, 540.0, nan, nan, nan, nan, nan, nan, 570.0, nan, nan, nan, nan, nan, nan, nan, nan, 600.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 840.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 720.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 780.0, nan, nan, nan, nan, 810.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, -720.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, -690.0, nan, nan, nan, nan, nan, nan, nan, -660.0, nan, nan, nan, nan, nan, -630.0, nan, nan, nan, nan, nan, nan, nan, nan, -600.0, nan, nan, -570.0, nan, nan, nan, nan, nan, nan, -540.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, -510.0, nan, nan, nan, -480.0, nan, nan, nan, nan, nan, nan, nan, -450.0, nan, nan, nan, nan, nan, nan, nan, nan, -420.0, nan, nan, nan, nan, -390.0, nan, nan, nan, nan, nan, nan, nan, -360.0, nan, nan, nan, nan, nan, -330.0, nan, nan, nan, nan, nan, 630.0, nan, nan, -300.0, nan, nan, nan, nan, nan, nan, -270.0, nan, nan, nan, nan, -240.0, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, -210.0, nan, nan, nan, nan, -180.0, -30.0, nan, nan, nan, nan, nan, -150.0, nan, nan, nan, 660.0, nan, nan, nan, -120.0, nan, nan, nan, nan, nan, nan, nan, -90.0, nan, nan, nan, nan, nan, nan, nan, -60.0, nan, nan, nan, nan, nan, nan, nan, nan, nan])\n"
     ]
    }
   ],
   "source": [
    "for col in train.columns:\n",
    "    if col != 'joinedAt':\n",
    "        print col + ':'\n",
    "        col_check(train, col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import hashlib\n",
    "import locale\n",
    "\n",
    "from collections import defaultdict\n",
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "# 特征编码处理类\n",
    "class FeatureEng:\n",
    "    def __init__(self):\n",
    "         # 载入 locales\n",
    "        self.localeIdMap = defaultdict(int)\n",
    "        for i, l in enumerate(locale.locale_alias.keys()):\n",
    "          self.localeIdMap[l] = i + 1\n",
    "        \n",
    "        # 载入 gender id 字典\n",
    "        ##缺失补0\n",
    "        self.genderIdMap = defaultdict(int, {'NaN': 0, \"male\":1, \"female\":2})\n",
    "        \n",
    "    def getData(self, col, data):\n",
    "        if col == 'locale':\n",
    "            return self.getLocaleId(data)\n",
    "        elif col == 'birthyear':\n",
    "            return self.getBirthYearInt(data)\n",
    "        elif col == 'gender':\n",
    "            return self.getGenderId(data)\n",
    "        elif col == 'joinedAt':\n",
    "            return self.getJoinedYearMonth(data)\n",
    "        elif col == 'timezone':\n",
    "            return self.getTimezoneInt(data)\n",
    "\n",
    "    def getLocaleId(self, locstr):\n",
    "        return self.localeIdMap[locstr.lower()]\n",
    "\n",
    "    def getGenderId(self, genderStr):\n",
    "        return self.genderIdMap[genderStr]\n",
    "\n",
    "    def getJoinedYearMonth(self, dateString):\n",
    "        try:\n",
    "            dttm = datetime.datetime.strptime(dateString, \"%Y-%m-%dT%H:%M:%S.%fZ\")\n",
    "            #return \"\".join([str(dttm.year), str(dttm.month)])\n",
    "            return (dttm.year-2010)*12 + dttm.month\n",
    "        except:  #缺失补0\n",
    "          return 0\n",
    "\n",
    "    def getBirthYearInt(self, birthYear):\n",
    "        #缺失补0\n",
    "        try:\n",
    "          return 0 if birthYear == \"None\" else int(birthYear)\n",
    "        except:\n",
    "          return 0\n",
    "\n",
    "    def getTimezoneInt(self, timezone):\n",
    "        try:\n",
    "          return int(timezone)\n",
    "        except:  #缺失值处理\n",
    "          return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对 nan 或者 None 值进行处理\n",
    "cols = train.columns\n",
    "n_cols = len(cols)\n",
    "userMatrix = np.zeros((train.shape[0],n_cols), dtype=np.int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         locale  birthyear    gender  joinedAt  timezone\n",
      "0      0.000020   0.000027  0.000019  0.000026  0.000036\n",
      "1      0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "2      0.000028   0.000027  0.000019  0.000026 -0.000018\n",
      "3      0.000028   0.000027  0.000038  0.000027  0.000016\n",
      "4      0.000020   0.000027  0.000038  0.000026  0.000031\n",
      "5      0.000045   0.000027  0.000038  0.000027  0.000018\n",
      "6      0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "7      0.000020   0.000027  0.000038  0.000026  0.000031\n",
      "8      0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "9      0.000020   0.000027  0.000038  0.000026  0.000031\n",
      "10     0.000028   0.000027  0.000019  0.000026  0.000031\n",
      "11     0.000028   0.000027  0.000038  0.000026  0.000031\n",
      "12     0.000020   0.000027  0.000038  0.000026  0.000031\n",
      "13     0.000028   0.000027  0.000038  0.000026  0.000018\n",
      "14     0.000000   0.000000  0.000019  0.000025 -0.000022\n",
      "15     0.000027   0.000027  0.000019  0.000026 -0.000018\n",
      "16     0.000000   0.000027  0.000019  0.000025 -0.000018\n",
      "17     0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "18     0.000020   0.000027  0.000019  0.000026  0.000020\n",
      "19     0.000028   0.000000  0.000019  0.000022 -0.000018\n",
      "20     0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "21     0.000072   0.000027  0.000038  0.000026  0.000018\n",
      "22     0.000028   0.000027  0.000019  0.000026  0.000031\n",
      "23     0.000028   0.000027  0.000038  0.000026  0.000031\n",
      "24     0.000020   0.000027  0.000019  0.000027 -0.000022\n",
      "25     0.000028   0.000027  0.000019  0.000023 -0.000018\n",
      "26     0.000020   0.000027  0.000019  0.000027 -0.000022\n",
      "27     0.000028   0.000027  0.000038  0.000026 -0.000018\n",
      "28     0.000028   0.000027  0.000038  0.000026  0.000031\n",
      "29     0.000028   0.000027  0.000019  0.000027 -0.000036\n",
      "...         ...        ...       ...       ...       ...\n",
      "38179  0.000000   0.000027  0.000019  0.000026 -0.000022\n",
      "38180  0.000020   0.000027  0.000019  0.000027  0.000031\n",
      "38181  0.000028   0.000027  0.000019  0.000027  0.000040\n",
      "38182  0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "38183  0.000028   0.000000  0.000019  0.000022 -0.000018\n",
      "38184  0.000028   0.000027  0.000038  0.000026  0.000031\n",
      "38185  0.000020   0.000027  0.000038  0.000026  0.000031\n",
      "38186  0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "38187  0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "38188  0.000028   0.000027  0.000019  0.000023 -0.000018\n",
      "38189  0.000004   0.000027  0.000038  0.000027  0.000004\n",
      "38190  0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "38191  0.000020   0.000027  0.000038  0.000026 -0.000031\n",
      "38192  0.000020   0.000027  0.000038  0.000026  0.000031\n",
      "38193  0.000000   0.000027  0.000019  0.000025  0.000013\n",
      "38194  0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "38195  0.000072   0.000027  0.000038  0.000025 -0.000022\n",
      "38196  0.000028   0.000027  0.000038  0.000026 -0.000018\n",
      "38197  0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "38198  0.000028   0.000027  0.000019  0.000027 -0.000036\n",
      "38199  0.000072   0.000027  0.000038  0.000027  0.000031\n",
      "38200  0.000031   0.000027  0.000019  0.000026 -0.000009\n",
      "38201  0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "38202  0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "38203  0.000020   0.000027  0.000038  0.000027  0.000031\n",
      "38204  0.000028   0.000027  0.000038  0.000023 -0.000018\n",
      "38205  0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "38206  0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "38207  0.000028   0.000027  0.000019  0.000026  0.000031\n",
      "38208  0.000028   0.000027  0.000019  0.000027 -0.000036\n",
      "\n",
      "[38209 rows x 5 columns]\n"
     ]
    }
   ],
   "source": [
    "FE = FeatureEng()\n",
    "\n",
    "for i in range(train.shape[0]): \n",
    "    for col, index in zip(cols,range(n_cols)):\n",
    "        userMatrix[i, index] = FE.getData(str(col), train.loc[i, col])\n",
    "\n",
    "# 归一化用户矩阵\n",
    "userMatrix = normalize(userMatrix, norm=\"l1\", axis=0, copy=False)\n",
    "\n",
    "df_FE = pd.DataFrame(data=userMatrix, columns=cols)\n",
    "print df_FE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 聚类、CH_Score计算、画图、保存结果到文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 聚类\n",
    "from sklearn.cluster import MiniBatchKMeans\n",
    "\n",
    "def mbkmeans(data, k):\n",
    "    print 'K-means:' + str(k)\n",
    "    return MiniBatchKMeans(n_clusters = k).fit_predict(data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics\n",
    "\n",
    "# 计算 ch_score\n",
    "def cal_ch_score(data, result):\n",
    "    CH_score = metrics.calinski_harabaz_score(data, result)\n",
    "    print 'CH_score:'+str(CH_score)\n",
    "    return CH_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 保存画图\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def save_plt(ks, scores):\n",
    "    print 'save_plt'\n",
    "    plt.plot(Ks, np.array(CH_scores), 'b-')\n",
    "    plt.savefig('result.png')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 保存结果到文件\n",
    "def save_data(data, result, k):\n",
    "    print 'save_data:' + str(k)\n",
    "    data['result_'+str(k)] = result\n",
    "    data.to_csv('result_' + str(k) + '.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     locale  birthyear    gender  joinedAt  timezone\n",
      "0  0.000020   0.000027  0.000019  0.000026  0.000036\n",
      "1  0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "2  0.000028   0.000027  0.000019  0.000026 -0.000018\n",
      "3  0.000028   0.000027  0.000038  0.000027  0.000016\n",
      "4  0.000020   0.000027  0.000038  0.000026  0.000031\n",
      "K-means:20\n",
      "CH_score:42994.3646324\n",
      "save_data:20\n",
      "     locale  birthyear    gender  joinedAt  timezone\n",
      "0  0.000020   0.000027  0.000019  0.000026  0.000036\n",
      "1  0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "2  0.000028   0.000027  0.000019  0.000026 -0.000018\n",
      "3  0.000028   0.000027  0.000038  0.000027  0.000016\n",
      "4  0.000020   0.000027  0.000038  0.000026  0.000031\n",
      "K-means:40\n",
      "CH_score:45238.9745061\n",
      "save_data:40\n",
      "     locale  birthyear    gender  joinedAt  timezone\n",
      "0  0.000020   0.000027  0.000019  0.000026  0.000036\n",
      "1  0.000020   0.000027  0.000019  0.000026  0.000031\n",
      "2  0.000028   0.000027  0.000019  0.000026 -0.000018\n",
      "3  0.000028   0.000027  0.000038  0.000027  0.000016\n",
      "4  0.000020   0.000027  0.000038  0.000026  0.000031\n",
      "K-means:80\n",
      "CH_score:38261.395359\n",
      "save_data:80\n",
      "save_plt\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cluster import MiniBatchKMeans\n",
    "\n",
    "# 尝试 K=20， 40， 80，并计算各自 CH_score\n",
    "Ks = [20, 40, 80]\n",
    "CH_scores = []\n",
    "\n",
    "for K in Ks:\n",
    "    train = df_FE.copy()\n",
    "    # 聚类\n",
    "    result = mbkmeans(train, K)\n",
    "    # CH_Score 计算\n",
    "    CH_scores.append(cal_ch_score(train, result))\n",
    "    # 保存文件\n",
    "    save_data(train, result, K)\n",
    "# 画图\n",
    "save_plt(Ks, CH_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
